<?php

declare(strict_types=1);

namespace app\model\crawler;

use think\facade\Cache;
use voku\helper\HtmlDomParser;
use app\eslib\EsUtils;

/**
 * 采集器
 */
class CrawlerModel
{
    public static function start()
    {

        $n = self::saveToCache('kuaikeji', 'https://www.mydrivers.com/', ['listSelector' => '.news_info a', 'detailSelector' => '.news_info', 'detailDelSelector' => [
            '.news_info div','.news_info .end', '.news_info .bqian'
        ]]);
        $n = self::saveToCache('oschina', 'https://www.oschina.net/', ['listSelector' => '#v_news .item-list a', 'detailSelector' => '.article-detail', 'detailDelSelector' => []]);
        return $n;
    }

    /**
     * 保存到本地缓存
     */
    public function saveToCache($siteId, $listUrl, $option = [])
    {
        $dir = runtime_path('crawler/list/');
        $list = $this->crawlListPage($siteId, $listUrl, $option);
        $data = [];
        $contentHtml = '';
        foreach ($list as $row) {
            $contentHtml = $this->crawlDetailPage($siteId, $row['link'], $option, $option);
            $data[] = [
                'title' => $row['text'],
                'category_id' => 0,
                'summary' => EsUtils::shortText(trim(strip_tags($contentHtml)), 150, 0),
                'content' => $contentHtml,
                'crawler_url' => $row['link'],
                'crawler_site' => 'kuaileji',
                'source_type' => 0,
                'author_id' => 0,
                'created_at' => time(),
            ];
        }
        return $data;
    }
    /**
     * 采集列表页
     */
    public function crawlListPage($siteId, $listUrl, $option = [])
    {
        $cacheKey = $siteId . '_list';
        $data = Cache::get($cacheKey);
        if ($data) {
            return $data;
        }
        $data = [];
        $dom = HtmlDomParser::file_get_html($listUrl);
        $elements = $dom->findMulti($option['listSelector']);
        foreach ($elements as $element) {
            $data[] = ['text' => $element->text(), 'link' => $element->getAttribute('href')];
        }
        unset($elements);
        unset($dom);
        Cache::set($cacheKey, $data, 3600);
        return $data;
    }

    /**
     * 采集详情页
     */
    public function crawlDetailPage($siteId, $detailUrl, $option = [])
    {
        $cacheKey = $siteId . "_detail" . substr(md5($detailUrl), 8, 16);
        $data = Cache::get($cacheKey);
        if ($data) {
            return $data;
        }
        $dom = HtmlDomParser::file_get_html($detailUrl);
        if ($option['detailDelSelector']) {
            // 清理广告信息等元素节点
            foreach ($option['detailDelSelector'] as $selector) {
                $nodes = $dom->findMultiOrFalse($selector);
                if ($nodes) {
                    foreach ($nodes as $node) {
                        $node->outertext = '';
                    }
                }
                unset($nodes);
            }
        }
        // 获取内容
        $data = $dom->findOne($option['detailSelector'])->html();
        unset($dom);
        Cache::set($cacheKey, $data, 3600);
        return $data;
    }
}
